{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "58da6cc5-dcd6-4bde-8393-91f7e6c52601",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-06T03:30:02.804934Z",
     "iopub.status.busy": "2022-07-06T03:30:02.804686Z",
     "iopub.status.idle": "2022-07-06T03:30:33.478641Z",
     "shell.execute_reply": "2022-07-06T03:30:33.477931Z",
     "shell.execute_reply.started": "2022-07-06T03:30:02.804907Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting raydp\n",
      "  Downloading raydp-0.4.2-py3-none-any.whl (10.5 MB)\n",
      "\u001b[K     |████████████████████████████████| 10.5 MB 31.5 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: psutil in /mnt/notebook-env/lib/python3.7/site-packages (from raydp) (5.9.1)\n",
      "Requirement already satisfied: numpy in /mnt/notebook-env/lib/python3.7/site-packages (from raydp) (1.21.2)\n",
      "Collecting pyspark>=3.2.0\n",
      "  Downloading pyspark-3.3.0.tar.gz (281.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 281.3 MB 60 kB/s /s eta 0:00:01\n",
      "\u001b[?25hCollecting typing\n",
      "  Downloading typing-3.7.4.3.tar.gz (78 kB)\n",
      "\u001b[K     |████████████████████████████████| 78 kB 19.6 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: ray[default]>=1.8.0 in /mnt/notebook-env/lib/python3.7/site-packages (from raydp) (1.13.0)\n",
      "Requirement already satisfied: pyarrow>=0.10 in /mnt/notebook-env/lib/python3.7/site-packages (from raydp) (6.0.1)\n",
      "Collecting aiohttp==3.7.4\n",
      "  Downloading aiohttp-3.7.4-cp37-cp37m-manylinux2014_x86_64.whl (1.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.3 MB 73.8 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting netifaces\n",
      "  Downloading netifaces-0.11.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (32 kB)\n",
      "Requirement already satisfied: pandas>=1.1.4 in /mnt/notebook-env/lib/python3.7/site-packages (from raydp) (1.3.5)\n",
      "Collecting chardet<4.0,>=2.0\n",
      "  Downloading chardet-3.0.4-py2.py3-none-any.whl (133 kB)\n",
      "\u001b[K     |████████████████████████████████| 133 kB 105.3 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting async-timeout<4.0,>=3.0\n",
      "  Downloading async_timeout-3.0.1-py3-none-any.whl (8.2 kB)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /mnt/notebook-env/lib/python3.7/site-packages (from aiohttp==3.7.4->raydp) (21.4.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /mnt/notebook-env/lib/python3.7/site-packages (from aiohttp==3.7.4->raydp) (5.2.0)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /mnt/notebook-env/lib/python3.7/site-packages (from aiohttp==3.7.4->raydp) (1.6.3)\n",
      "Requirement already satisfied: typing-extensions>=3.6.5 in /mnt/notebook-env/lib/python3.7/site-packages (from aiohttp==3.7.4->raydp) (4.1.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /mnt/notebook-env/lib/python3.7/site-packages (from pandas>=1.1.4->raydp) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /mnt/notebook-env/lib/python3.7/site-packages (from pandas>=1.1.4->raydp) (2021.3)\n",
      "Collecting py4j==0.10.9.5\n",
      "  Downloading py4j-0.10.9.5-py2.py3-none-any.whl (199 kB)\n",
      "\u001b[K     |████████████████████████████████| 199 kB 101.4 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: six>=1.5 in /mnt/notebook-env/lib/python3.7/site-packages (from python-dateutil>=2.7.3->pandas>=1.1.4->raydp) (1.16.0)\n",
      "Requirement already satisfied: pyyaml in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (6.0)\n",
      "Requirement already satisfied: virtualenv in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (20.15.1)\n",
      "Requirement already satisfied: protobuf<4.0.0,>=3.15.3 in /home/emr-notebook/.local/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (3.19.4)\n",
      "Requirement already satisfied: grpcio<=1.43.0,>=1.28.1 in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (1.43.0)\n",
      "Requirement already satisfied: click<=8.0.4,>=7.0 in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (8.0.4)\n",
      "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (1.0.4)\n",
      "Requirement already satisfied: filelock in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (3.7.1)\n",
      "Requirement already satisfied: requests in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (2.27.1)\n",
      "Requirement already satisfied: frozenlist in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (1.2.0)\n",
      "Requirement already satisfied: jsonschema in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (3.2.0)\n",
      "Requirement already satisfied: aiosignal in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (1.2.0)\n",
      "Requirement already satisfied: opencensus in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (0.10.0)\n",
      "Requirement already satisfied: colorful in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (0.5.4)\n",
      "Requirement already satisfied: aiohttp-cors in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (0.7.0)\n",
      "Requirement already satisfied: prometheus-client<0.14.0,>=0.7.1 in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (0.13.1)\n",
      "Requirement already satisfied: gpustat>=1.0.0b1 in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (1.0.0rc1)\n",
      "Requirement already satisfied: smart-open in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (6.0.0)\n",
      "Requirement already satisfied: py-spy>=0.2.0 in /mnt/notebook-env/lib/python3.7/site-packages (from ray[default]>=1.8.0->raydp) (0.3.12)\n",
      "Requirement already satisfied: importlib-metadata in /mnt/notebook-env/lib/python3.7/site-packages (from click<=8.0.4,>=7.0->ray[default]>=1.8.0->raydp) (4.8.2)\n",
      "Requirement already satisfied: blessed>=1.17.1 in /mnt/notebook-env/lib/python3.7/site-packages (from gpustat>=1.0.0b1->ray[default]>=1.8.0->raydp) (1.19.1)\n",
      "Requirement already satisfied: nvidia-ml-py<=11.495.46,>=11.450.129 in /mnt/notebook-env/lib/python3.7/site-packages (from gpustat>=1.0.0b1->ray[default]>=1.8.0->raydp) (11.495.46)\n",
      "Requirement already satisfied: wcwidth>=0.1.4 in /mnt/notebook-env/lib/python3.7/site-packages (from blessed>=1.17.1->gpustat>=1.0.0b1->ray[default]>=1.8.0->raydp) (0.2.5)\n",
      "Requirement already satisfied: idna>=2.0 in /mnt/notebook-env/lib/python3.7/site-packages (from yarl<2.0,>=1.0->aiohttp==3.7.4->raydp) (3.3)\n",
      "Requirement already satisfied: zipp>=0.5 in /mnt/notebook-env/lib/python3.7/site-packages (from importlib-metadata->click<=8.0.4,>=7.0->ray[default]>=1.8.0->raydp) (3.7.0)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /mnt/notebook-env/lib/python3.7/site-packages (from jsonschema->ray[default]>=1.8.0->raydp) (0.18.0)\n",
      "Requirement already satisfied: setuptools in /mnt/notebook-env/lib/python3.7/site-packages (from jsonschema->ray[default]>=1.8.0->raydp) (58.0.4)\n",
      "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /mnt/notebook-env/lib/python3.7/site-packages (from opencensus->ray[default]>=1.8.0->raydp) (2.8.2)\n",
      "Requirement already satisfied: opencensus-context>=0.1.2 in /mnt/notebook-env/lib/python3.7/site-packages (from opencensus->ray[default]>=1.8.0->raydp) (0.1.2)\n",
      "Requirement already satisfied: google-auth<3.0dev,>=1.25.0 in /mnt/notebook-env/lib/python3.7/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]>=1.8.0->raydp) (2.6.0)\n",
      "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.56.2 in /mnt/notebook-env/lib/python3.7/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]>=1.8.0->raydp) (1.56.3)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /mnt/notebook-env/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]>=1.8.0->raydp) (4.2.2)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /mnt/notebook-env/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]>=1.8.0->raydp) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /mnt/notebook-env/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]>=1.8.0->raydp) (4.7.2)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /mnt/notebook-env/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0dev,>=1.25.0->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]>=1.8.0->raydp) (0.4.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /mnt/notebook-env/lib/python3.7/site-packages (from requests->ray[default]>=1.8.0->raydp) (2022.6.15)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /mnt/notebook-env/lib/python3.7/site-packages (from requests->ray[default]>=1.8.0->raydp) (1.25.11)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /mnt/notebook-env/lib/python3.7/site-packages (from requests->ray[default]>=1.8.0->raydp) (2.0.4)\n",
      "Requirement already satisfied: distlib<1,>=0.3.1 in /mnt/notebook-env/lib/python3.7/site-packages (from virtualenv->ray[default]>=1.8.0->raydp) (0.3.4)\n",
      "Requirement already satisfied: platformdirs<3,>=2 in /mnt/notebook-env/lib/python3.7/site-packages (from virtualenv->ray[default]>=1.8.0->raydp) (2.5.2)\n",
      "Building wheels for collected packages: pyspark, typing\n",
      "  Building wheel for pyspark (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for pyspark: filename=pyspark-3.3.0-py2.py3-none-any.whl size=281764026 sha256=7fc4fca498a6f1a203620531706206e5f2cfae5ab5c5a01d9763de43162d4dca\n",
      "  Stored in directory: /home/emr-notebook/.cache/pip/wheels/7a/8e/1b/f73a52650d2e5f337708d9f6a1750d451a7349a867f928b885\n",
      "  Building wheel for typing (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for typing: filename=typing-3.7.4.3-py3-none-any.whl size=26325 sha256=54f6c20944ca7ba720c118e988145446231e6f4df44e27b4b56eeec52a93c2f6\n",
      "  Stored in directory: /home/emr-notebook/.cache/pip/wheels/35/f3/15/01aa6571f0a72ee6ae7b827c1491c37a1f72d686fd22b43b0e\n",
      "Successfully built pyspark typing\n",
      "Installing collected packages: chardet, async-timeout, aiohttp, py4j, typing, pyspark, netifaces, raydp\n",
      "\u001b[33m  WARNING: The script chardetect is installed in '/home/emr-notebook/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
      "Successfully installed aiohttp-3.7.4 async-timeout-3.0.1 chardet-3.0.4 netifaces-0.11.0 py4j-0.10.9.5 pyspark-3.3.0 raydp-0.4.2 typing-3.7.4.3\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install raydp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "26cca7ec-08ca-4109-8d46-129ec253627d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-06T03:31:31.187718Z",
     "iopub.status.busy": "2022-07-06T03:31:31.187479Z",
     "iopub.status.idle": "2022-07-06T03:31:31.197567Z",
     "shell.execute_reply": "2022-07-06T03:31:31.197003Z",
     "shell.execute_reply.started": "2022-07-06T03:31:31.187694Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'status': 'ok', 'restart': True}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import IPython\n",
    "\n",
    "IPython.Application.instance().kernel.do_shutdown(True) #automatically restarts kernel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f4f2c79c-24b3-4fb7-b109-18311381966a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-06T12:48:43.998400Z",
     "iopub.status.busy": "2022-07-06T12:48:43.998123Z",
     "iopub.status.idle": "2022-07-06T12:48:48.333942Z",
     "shell.execute_reply": "2022-07-06T12:48:48.333282Z",
     "shell.execute_reply.started": "2022-07-06T12:48:43.998328Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'NoneType' object has no attribute 'node_ip_address'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/raydp/context.py\u001b[0m in \u001b[0;36minit_spark\u001b[0;34m(app_name, num_executors, executor_cores, executor_memory, configs)\u001b[0m\n\u001b[1;32m    124\u001b[0m                 _global_spark_context = _SparkContext(\n\u001b[1;32m    125\u001b[0m                     app_name, num_executors, executor_cores, executor_memory, configs)\n\u001b[0;32m--> 126\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0m_global_spark_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_or_create_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    127\u001b[0m             \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m                 \u001b[0m_global_spark_context\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/raydp/context.py\u001b[0m in \u001b[0;36mget_or_create_session\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     68\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_spark_session\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRayDPConversionHelper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mRAYDP_OBJ_HOLDER_NAME\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mremote\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m         \u001b[0mspark_cluster\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_or_create_spark_cluster\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     71\u001b[0m         self._spark_session = spark_cluster.get_spark_session(\n\u001b[1;32m     72\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_app_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/raydp/context.py\u001b[0m in \u001b[0;36m_get_or_create_spark_cluster\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     61\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_spark_cluster\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_spark_cluster\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_spark_cluster\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSparkCluster\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_configs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     64\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_spark_cluster\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/raydp/spark/ray_cluster.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, configs)\u001b[0m\n\u001b[1;32m     32\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_app_master_bridge\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_configs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfigs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_up_master\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_spark_session\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mSparkSession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/raydp/spark/ray_cluster.py\u001b[0m in \u001b[0;36m_set_up_master\u001b[0;34m(self, resources, kwargs)\u001b[0m\n\u001b[1;32m     38\u001b[0m         \u001b[0;31m# TODO: specify the app master resource\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_app_master_bridge\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRayClusterMaster\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_configs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_app_master_bridge\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_up\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_set_up_worker\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresources\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfloat\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/raydp/spark/ray_cluster_master.py\u001b[0m in \u001b[0;36mstart_up\u001b[0;34m(self, popen_kwargs)\u001b[0m\n\u001b[1;32m     54\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gateway\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_launch_gateway\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mextra_classpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpopen_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_app_master_java_bridge\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gateway\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mentry_point\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetAppMasterBridge\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_properties\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     57\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_host\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_node_ip_address\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_app_master\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mextra_classpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/raydp/spark/ray_cluster_master.py\u001b[0m in \u001b[0;36m_set_properties\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m         \u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"ray.run-mode\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"CLUSTER\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m         \u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"ray.node-ip\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnode_ip_address\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    146\u001b[0m         \u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"ray.address\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m         \u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"ray.redis.password\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mredis_password\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'node_ip_address'"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "import ray\n",
    "import raydp\n",
    "\n",
    "ray.shutdown()\n",
    "\n",
    "head_service_ip = \"localhost\"\n",
    "client_port = 10001\n",
    "ray.util.connect(f\"{head_service_ip}:{client_port}\")\n",
    "#                 runtime_env={\"pip\": [\"raydp\"]})\n",
    "\n",
    "# ray.init(address=\"ray://localhost:10001\",\n",
    "#          runtime_env={\"pip\": [\"raydp\"]})\n",
    "\n",
    "configs={\n",
    "    \"spark.driver.extraJavaOptions\": \"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED\",\n",
    "#    \"spark.jars.packages\": \"org.apache.hadoop:hadoop-aws:3.2.0,com.amazonaws:aws-java-sdk-s3:1.12.210,com.amazonaws:aws-java-sdk:1.12.210\",\n",
    "#    \"spark.hadoop.fs.s3a.impl\": \"org.apache.hadoop.fs.s3a.S3AFileSystem\",\n",
    "#    \"spark.hadoop.fs.s3a.aws.credentials.provider\": \"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider\"\n",
    "}\n",
    "\n",
    "raydp.stop_spark()\n",
    "spark = raydp.init_spark(\n",
    "  app_name = \"example\",\n",
    "  num_executors = 1,\n",
    "  executor_cores = 1,\n",
    "  executor_memory = \"1GB\",\n",
    "  configs = configs\n",
    ")\n",
    "\n",
    "df_from_csv = spark.read.option('delimiter', ',') \\\n",
    "                        .option('header', True) \\\n",
    "                        .csv('./data/train/part-algo-1-womens_clothing_ecommerce_reviews.csv')\n",
    "#                        .csv('s3a://dsoaws/ray/datasets/data/train/part-algo-1-womens_clothing_ecommerce_reviews.csv')\n",
    "\n",
    "\n",
    "print(df_from_csv)\n",
    "\n",
    "df_from_csv.groupBy(\"sentiment\").count().show()\n",
    "\n",
    "raydp.stop_spark()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4fe345a-d1fe-4981-8368-07c177806a58",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
